"""
@version: python3
#@time: 2021/06/23
@file: weather.py
@time: 2021/6/23 下午2:44
@ 主要爬取了天气的数据
"""

import pandas as pd
import requests
import demjson
import datetime
from dateutil.relativedelta import relativedelta
from retrying import retry
from tqdm import tqdm
import time
import random
from data import city_code
USER_AGENT_LIST=[
                       'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36',
                       'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36',
                       'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36',
                       'Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36',
                       'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36',
                       'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36',
                       'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0) Gecko/20100101 Firefox/17.0.6',
                       'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36',
                       'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36',
                       'Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36'
]

def get_data(city_code):
    data =[]
    for line in city_code.split('|'):
        pro_code = line.split(' ')[0].split('-')[0]
        cide_V = line.split(' ')[0].split('-')[1]
        city = line.split(' ')[1].split('-')[0]
        city_code = line.split(' ')[1].split('-')[1]
        data.append([pro_code,cide_V,city,city_code])
    return data

def get_city():
    data = get_data(city_code)
    data =pd.DataFrame(data,columns =['city_code','cide_V','city_name','area_code'])
    datax =data[data.city_code ==data.area_code]
    datay = datax[['area_code','city_name']]
    datay.columns =['area_code','area_name']
    dataz= data.merge(datay,on='area_code')
    return dataz


class Weather:
    def __init__(self, city_name,city_code,area_name,log_flag=False):
        self.city = city_name
        self.city_code = city_code
        self.area_name =area_name
        self.log_flag = log_flag

    def log(self, *args):
        if self.log_flag:
            print(*args)

    def save_weather(self, start, end):
        df = self.read_weather(start, end).dropna().reset_index()
        print(len(df))
        print(df['ymd'])



    @classmethod
    def date_generate(cls, start, end):
        start = datetime.datetime.strptime(start, '%Y%m')
        end = datetime.datetime.strptime(end, '%Y%m')
        while True:
            next_start = start + relativedelta(months=1)
            yield start.strftime('%Y%m')
            if next_start > end:
                break
            start = next_start

    def read_weather(self, start, end):
        dates = self.date_generate(start, end)
        return pd.concat(map(self.get_weather, dates))

    def get_url(self, date):
        url = 'https://tianqi.2345.com/t/wea_history/js/{date}/{city_code}_{date}.js'.format(
            date=date, city_code=self.city_code)
        return url
    @retry(stop_max_attempt_number=3)
    def get_weather_json(self, url):
        print(url)
        self.log(url)
        header = {
            'User-agent': random.choice(USER_AGENT_LIST),
        }
        weather = requests.get(url,headers=header).text.split('=')[1][:-1]
        weather = demjson.decode(weather)['tqInfo']
        return weather

    def get_weather(self, date):
        url = self.get_url(date)
        weather = self.get_weather_json(url)
        df = pd.DataFrame(weather)
        df['month_index'] = date
        df['city_code'] =self.city_code
        df['city_name'] = self.city
        df['area_name'] =self.area_name
        today_df = df.loc[df['ymd']==date_today]
        choice = 4
        if choice ==1:return df
        elif choice == 2:return today_df
        #默认返回
        return df
def main():
    data = get_city()
    endmonth = datetime.date.today().strftime('%Y%m')  # 当月
    date_today = datetime.date.today().strftime('%Y-%m-%d')  # 当日
    beginmonth = (datetime.datetime.now().date() - relativedelta(months=1)).strftime('%Y%m')  # 上个月

    for i in tqdm(range(len(data))):  # len(data)
        # 如果代码中断，看 i 到哪里了，继续重新跑：for i in tqdm(range(len(data[i+200:])))
        # i =i+200
        if (i % 50 == 0) & (i != 0):  # 跑个50个，休息个120s ，否则容易被封
            time.sleep(120)
        else:
            time.sleep(1)
        city_code = data['city_code'][i]
        city_name = data['city_name'][i]
        area_name = data['area_name'][i]

        df = Weather(city_name, city_code, area_name).read_weather(beginmonth, endmonth).dropna().reset_index()
        print(len(df))
        print(df['ymd'])


if __name__ == '__main__':
    # main()

    #test
    data =get_city()
    endmonth = datetime.date.today().strftime('%Y%m')  # 当月
    date_today = datetime.date.today().strftime('%Y-%m-%d')  # 当日
    beginmonth = (datetime.datetime.now().date() - relativedelta(months=1)).strftime('%Y%m')  # 上个月

    for i in tqdm(range(len(data)-len(data)+2)): #len(data)
        # 如果代码中断，看 i 到哪里了，继续重新跑：for i in tqdm(range(len(data[i+200:])))
        # i =i+200
        if i%50==49: # 跑个50个，休息个120s ，否则容易被封
            time.sleep(120)
        else:
            time.sleep(1)
        city_code = data['city_code'][i]
        city_name = data['city_name'][i]
        area_name = data['area_name'][i]

        df = Weather(city_name,city_code,area_name).read_weather(beginmonth,endmonth).dropna().reset_index()
        print(len(df))
        print(df['ymd'])
